Merge pull request #623 from dsander/s3-bundle-cache
'Manually' cache bundle with S3
Dominik Sander
лет %!s(int64=10): %!d(string=назад)
3 измененных файлов с 134 добавлено и 15 удалено
+ 16
- 15
.travis.yml
@@ -1,25 +1,26 @@ |
||
| 1 | 1 |
language: ruby |
| 2 |
-bundler_args: --without development production --deployment |
|
| 3 |
-cache: |
|
| 4 |
- directories: |
|
| 5 |
- - vendor/bundle |
|
| 6 | 2 |
env: |
| 3 |
+ matrix: |
|
| 7 | 4 |
- APP_SECRET_TOKEN=b2724973fd81c2f4ac0f92ac48eb3f0152c4a11824c122bcf783419a4c51d8b9bba81c8ba6a66c7de599677c7f486242cf819775c433908e77c739c5c8ae118d |
| 5 |
+ global: |
|
| 6 |
+ - AMAZON_S3_BUCKET=huginn-bundle-cache |
|
| 7 |
+ - AMAZON_ACCESS_KEY_ID=AKIAITMDBT3BL3ZWVFGA |
|
| 8 | 8 |
rvm: |
| 9 |
- - 2.0.0 |
|
| 10 |
- - 2.1.1 |
|
| 11 |
- - 1.9.3 |
|
| 9 |
+- 2.0.0 |
|
| 10 |
+- 2.1.4 |
|
| 11 |
+- 1.9.3 |
|
| 12 | 12 |
before_install: |
| 13 |
- - travis_retry gem install bundler |
|
| 13 |
+- travis_retry gem install bundler |
|
| 14 |
+install: travis_retry script/cached-bundle install --without development production --deployment |
|
| 14 | 15 |
before_script: |
| 15 |
- - mysql -e 'create database huginn_test;' |
|
| 16 |
- - bundle exec rake db:migrate db:test:prepare |
|
| 17 |
-script: "bundle exec rake" |
|
| 16 |
+- mysql -e 'create database huginn_test;' |
|
| 17 |
+- bundle exec rake db:migrate db:test:prepare |
|
| 18 |
+script: bundle exec rake |
|
| 18 | 19 |
notifications: |
| 19 | 20 |
irc: |
| 20 | 21 |
channels: |
| 21 |
- - "chat.freenode.net#huginn" |
|
| 22 |
+ - chat.freenode.net#huginn |
|
| 22 | 23 |
template: |
| 23 |
- - "<%{author}> %{branch} - %{commit} (%{commit_message}): %{message}"
|
|
| 24 |
- - "Change view : %{compare_url}"
|
|
| 25 |
- - "Build details : %{build_url}"
|
|
| 24 |
+ - "<%{author}> %{branch} - %{commit} (%{commit_message}): %{message}"
|
|
| 25 |
+ - 'Change view : %{compare_url}'
|
|
| 26 |
+ - 'Build details : %{build_url}'
|
+ 47
- 0
script/cached-bundle
@@ -0,0 +1,47 @@ |
||
| 1 |
+#!/usr/bin/env bash |
|
| 2 |
+# Usage: cached-bundle install --deployment |
|
| 3 |
+# |
|
| 4 |
+# After running `bundle`, caches the `vendor/bundle` directory to S3. |
|
| 5 |
+# On the next run, restores the cached directory before running `bundle`. |
|
| 6 |
+# When `Gemfile.lock` changes, the cache gets rebuilt. |
|
| 7 |
+# |
|
| 8 |
+# Requirements: |
|
| 9 |
+# - Gemfile.lock |
|
| 10 |
+# - TRAVIS_REPO_SLUG |
|
| 11 |
+# - TRAVIS_RUBY_VERSION |
|
| 12 |
+# - AMAZON_S3_BUCKET |
|
| 13 |
+# - script/s3-put |
|
| 14 |
+# - bundle |
|
| 15 |
+# - curl |
|
| 16 |
+# |
|
| 17 |
+# Author: Mislav Marohnić |
|
| 18 |
+ |
|
| 19 |
+set -e |
|
| 20 |
+ |
|
| 21 |
+compute_md5() {
|
|
| 22 |
+ local output="$(openssl md5)" |
|
| 23 |
+ echo "${output##* }"
|
|
| 24 |
+} |
|
| 25 |
+ |
|
| 26 |
+download() {
|
|
| 27 |
+ curl --tcp-nodelay -qsfL "$1" -o "$2" |
|
| 28 |
+} |
|
| 29 |
+ |
|
| 30 |
+bundle_path="vendor/bundle" |
|
| 31 |
+gemfile_hash="$(compute_md5 <"${BUNDLE_GEMFILE:-Gemfile}.lock")"
|
|
| 32 |
+#cache_name="${TRAVIS_RUBY_VERSION}-${gemfile_hash}.tgz"
|
|
| 33 |
+cache_name="${TRAVIS_RUBY_VERSION}.tgz"
|
|
| 34 |
+fetch_url="http://${AMAZON_S3_BUCKET}.s3.amazonaws.com/${cache_name}"
|
|
| 35 |
+ |
|
| 36 |
+if download "$fetch_url" "$cache_name"; then |
|
| 37 |
+ echo "Reusing cached bundle ${cache_name}"
|
|
| 38 |
+ tar xzf "$cache_name" |
|
| 39 |
+fi |
|
| 40 |
+ |
|
| 41 |
+bundle "$@" |
|
| 42 |
+ |
|
| 43 |
+if [ ! -f "$cache_name" ]; then |
|
| 44 |
+ echo "Caching \`${bundle_path}' to S3"
|
|
| 45 |
+ tar czf "$cache_name" "$bundle_path" |
|
| 46 |
+ script/s3-put "$cache_name" "${AMAZON_S3_BUCKET}:${cache_name}"
|
|
| 47 |
+fi |
+ 71
- 0
script/s3-put
@@ -0,0 +1,71 @@ |
||
| 1 |
+#!/usr/bin/env bash |
|
| 2 |
+# Usage: s3-put <FILE> <S3_BUCKET>[:<PATH>] [<CONTENT_TYPE>] |
|
| 3 |
+# |
|
| 4 |
+# Uploads a file to the Amazon S3 service. |
|
| 5 |
+# Outputs the URL for the newly uploaded file. |
|
| 6 |
+# |
|
| 7 |
+# Requirements: |
|
| 8 |
+# - AMAZON_ACCESS_KEY_ID |
|
| 9 |
+# - AMAZON_SECRET_ACCESS_KEY |
|
| 10 |
+# - openssl |
|
| 11 |
+# - curl |
|
| 12 |
+# |
|
| 13 |
+# Author: Mislav Marohnić |
|
| 14 |
+ |
|
| 15 |
+set -e |
|
| 16 |
+ |
|
| 17 |
+authorization() {
|
|
| 18 |
+ local signature="$(string_to_sign | hmac_sha1 | base64)" |
|
| 19 |
+ echo "AWS ${AMAZON_ACCESS_KEY_ID?}:${signature}"
|
|
| 20 |
+} |
|
| 21 |
+ |
|
| 22 |
+hmac_sha1() {
|
|
| 23 |
+ openssl dgst -binary -sha1 -hmac "${AMAZON_SECRET_ACCESS_KEY?}"
|
|
| 24 |
+} |
|
| 25 |
+ |
|
| 26 |
+base64() {
|
|
| 27 |
+ openssl enc -base64 |
|
| 28 |
+} |
|
| 29 |
+ |
|
| 30 |
+bin_md5() {
|
|
| 31 |
+ openssl dgst -binary -md5 |
|
| 32 |
+} |
|
| 33 |
+ |
|
| 34 |
+string_to_sign() {
|
|
| 35 |
+ echo "$http_method" |
|
| 36 |
+ echo "$content_md5" |
|
| 37 |
+ echo "$content_type" |
|
| 38 |
+ echo "$date" |
|
| 39 |
+ echo "x-amz-acl:$acl" |
|
| 40 |
+ printf "/$bucket/$remote_path" |
|
| 41 |
+} |
|
| 42 |
+ |
|
| 43 |
+date_string() {
|
|
| 44 |
+ LC_TIME=C date "+%a, %d %h %Y %T %z" |
|
| 45 |
+} |
|
| 46 |
+ |
|
| 47 |
+file="$1" |
|
| 48 |
+bucket="${2%%:*}"
|
|
| 49 |
+remote_path="${2#*:}"
|
|
| 50 |
+content_type="$3" |
|
| 51 |
+ |
|
| 52 |
+if [ -z "$remote_path" ] || [ "$remote_path" = "$bucket" ]; then |
|
| 53 |
+ remote_path="${file##*/}"
|
|
| 54 |
+fi |
|
| 55 |
+ |
|
| 56 |
+http_method=PUT |
|
| 57 |
+acl="public-read" |
|
| 58 |
+content_md5="$(bin_md5 < "$file" | base64)" |
|
| 59 |
+date="$(date_string)" |
|
| 60 |
+ |
|
| 61 |
+url="https://$bucket.s3.amazonaws.com/$remote_path" |
|
| 62 |
+ |
|
| 63 |
+curl -qsSf -T "$file" \ |
|
| 64 |
+ -H "Authorization: $(authorization)" \ |
|
| 65 |
+ -H "x-amz-acl: $acl" \ |
|
| 66 |
+ -H "Date: $date" \ |
|
| 67 |
+ -H "Content-MD5: $content_md5" \ |
|
| 68 |
+ -H "Content-Type: $content_type" \ |
|
| 69 |
+ "$url" |
|
| 70 |
+ |
|
| 71 |
+echo "$url" |